Integration¶

Sanorama Hie et al., 2019
GitHub
Tutorial external API
External external API tutorial

A fix to run scran pooling normalization computeSumFactors in current python environment.

In [1]:
import scanpy as sc
import scanorama

import numpy as np
import pandas as pd

import os
In [2]:
# Working directory 
os.chdir('/research/peer/fdeckert/FD20200109SPLENO')
In [3]:
# rpy2 
os.environ['R_HOME'] = '/home/fdeckert/bin/miniconda3/envs/p.3.8.12-FD20200109SPLENO/lib/R'
In [4]:
# Plotting 
import rpy2.robjects as robjects
color_load = robjects.r.source('plotting_global.R')
color = dict()
for i in range(len(color_load[0])):
    color[color_load[0].names[i]] = {key : color_load[0][i].rx2(key)[0] for key in color_load[0][i].names}

sc.set_figure_params(figsize=(5, 5))

Scanorama (SCTransform)¶

In [5]:
adata = sc.read_h5ad('data/object/so_sct.h5ad')
In [6]:
def set_color(categories): 
    
    categories = [x for x in categories if x in list(adata.obs.columns)]

    for category in categories: 
        
        adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
        
        keys = list(color[category].keys())
        keys = [x for x in keys if x in list(adata.obs[category])]

        adata.obs[category] = adata.obs[category].cat.reorder_categories(keys)
        adata.uns[category+'_colors'] = np.array([color[category].get(key) for key in keys], dtype=object)
        
# Set colors
set_color(list(color.keys()))
In [7]:
adata_sub = dict()
for sample_group in adata.obs['treatment'].unique():
    adata_sub[sample_group] = adata[adata.obs['treatment']==sample_group].copy()
adata_sub = list(adata_sub.values())
In [8]:
# Run Scanorama
scanorama.integrate_scanpy(adata_sub, dimred=200, knn=50, verbose=True)

# Concatenate scanorama output 
X_scanorama = [ad.obsm['X_scanorama'] for ad in adata_sub]
X_scanorama = np.concatenate(X_scanorama)

obs_names = [ad.obs_names for ad in adata_sub]
obs_names = np.concatenate(obs_names)
all(obs_names==adata.obs_names)

# Add X_scanorama integration to adata 
adata.obsm["X_scanorama"] = X_scanorama

Found 8000 genes among all datasets
Processing datasets (0, 1)
In [9]:
# # Dimensional reduction and clustering 
sc.pp.neighbors(adata, n_neighbors=100, use_rep='X_scanorama')
sc.tl.leiden(adata, resolution=1)
sc.tl.louvain(adata, resolution=1)
sc.tl.umap(adata)

# Plot 
sc.pl.umap(adata, color=['louvain', 'leiden', 'tissue', 'treatment', 'label_fine_haemosphere', 'sample_rep', 'cc_phase_class', 'pHb_RNA', 'pRb_RNA'], wspace=0.5, ncols=3)

Scanorama (SCTransform + 8000 HVG)¶

In [10]:
adata = sc.read_h5ad('data/object/so_sct.h5ad')
In [11]:
hvg_8000 = list(adata.uns['hvg_int_8000'])
adata = adata[:,hvg_8000]
In [12]:
def set_color(categories): 
    
    categories = [x for x in categories if x in list(adata.obs.columns)]

    for category in categories: 
        
        adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
        
        keys = list(color[category].keys())
        keys = [x for x in keys if x in list(adata.obs[category])]

        adata.obs[category] = adata.obs[category].cat.reorder_categories(keys)
        adata.uns[category+'_colors'] = np.array([color[category].get(key) for key in keys], dtype=object)
        
# Set colors
set_color(list(color.keys()))

/tmp/ipykernel_3028845/1707382715.py:7: ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.
  adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
In [13]:
adata_sub = dict()
for sample_group in adata.obs['treatment'].unique():
    adata_sub[sample_group] = adata[adata.obs['treatment']==sample_group].copy()
adata_sub = list(adata_sub.values())
In [14]:
# Run Scanorama
scanorama.integrate_scanpy(adata_sub, dimred=200, knn=50, verbose=True)

# Concatenate scanorama output 
X_scanorama = [ad.obsm['X_scanorama'] for ad in adata_sub]
X_scanorama = np.concatenate(X_scanorama)

obs_names = [ad.obs_names for ad in adata_sub]
obs_names = np.concatenate(obs_names)
all(obs_names==adata.obs_names)

# Add X_scanorama integration to adata 
adata.obsm["X_scanorama"] = X_scanorama

Found 8000 genes among all datasets
Processing datasets (0, 1)
In [15]:
# # Dimensional reduction and clustering 
sc.pp.neighbors(adata, n_neighbors=100, use_rep='X_scanorama')
sc.tl.leiden(adata, resolution=1)
sc.tl.louvain(adata, resolution=1)
sc.tl.umap(adata)

# Plot 
sc.pl.umap(adata, color=['louvain', 'leiden', 'tissue', 'treatment', 'label_fine_haemosphere', 'sample_rep', 'cc_phase_class', 'pHb_RNA', 'pRb_RNA'], wspace=0.5, ncols=3)

Scanorama (SCTransform + 4000 HVG)¶

In [16]:
adata = sc.read_h5ad('data/object/so_sct.h5ad')
In [17]:
hvg_4000 = list(adata.uns['hvg_int_4000'])
adata = adata[:,hvg_4000]
In [18]:
def set_color(categories): 
    
    categories = [x for x in categories if x in list(adata.obs.columns)]

    for category in categories: 
        
        adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
        
        keys = list(color[category].keys())
        keys = [x for x in keys if x in list(adata.obs[category])]

        adata.obs[category] = adata.obs[category].cat.reorder_categories(keys)
        adata.uns[category+'_colors'] = np.array([color[category].get(key) for key in keys], dtype=object)
        
# Set colors
set_color(list(color.keys()))

/tmp/ipykernel_3028845/1707382715.py:7: ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.
  adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
In [19]:
adata_sub = dict()
for sample_group in adata.obs['treatment'].unique():
    adata_sub[sample_group] = adata[adata.obs['treatment']==sample_group].copy()
adata_sub = list(adata_sub.values())
In [20]:
# Run Scanorama
scanorama.integrate_scanpy(adata_sub, dimred=200, knn=50, verbose=True)

# Concatenate scanorama output 
X_scanorama = [ad.obsm['X_scanorama'] for ad in adata_sub]
X_scanorama = np.concatenate(X_scanorama)

obs_names = [ad.obs_names for ad in adata_sub]
obs_names = np.concatenate(obs_names)
all(obs_names==adata.obs_names)

# Add X_scanorama integration to adata 
adata.obsm["X_scanorama"] = X_scanorama

Found 4000 genes among all datasets
Processing datasets (0, 1)
In [21]:
# # Dimensional reduction and clustering 
sc.pp.neighbors(adata, n_neighbors=100, use_rep='X_scanorama')
sc.tl.leiden(adata, resolution=1)
sc.tl.louvain(adata, resolution=1)
sc.tl.umap(adata)

# Plot 
sc.pl.umap(adata, color=['louvain', 'leiden', 'tissue', 'treatment', 'label_fine_haemosphere', 'sample_rep', 'cc_phase_class', 'pHb_RNA', 'pRb_RNA'], wspace=0.5, ncols=3)

Scanorama (SCTransform + 2000 HVG)¶

In [22]:
adata = sc.read_h5ad('data/object/so_sct.h5ad')
In [23]:
hvg_2000 = list(adata.uns['hvg_int_2000'])
adata = adata[:,hvg_2000]
In [24]:
def set_color(categories): 
    
    categories = [x for x in categories if x in list(adata.obs.columns)]

    for category in categories: 
        
        adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
        
        keys = list(color[category].keys())
        keys = [x for x in keys if x in list(adata.obs[category])]

        adata.obs[category] = adata.obs[category].cat.reorder_categories(keys)
        adata.uns[category+'_colors'] = np.array([color[category].get(key) for key in keys], dtype=object)
        
# Set colors
set_color(list(color.keys()))

/tmp/ipykernel_3028845/1707382715.py:7: ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.
  adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
In [25]:
adata_sub = dict()
for sample_group in adata.obs['treatment'].unique():
    adata_sub[sample_group] = adata[adata.obs['treatment']==sample_group].copy()
adata_sub = list(adata_sub.values())
In [26]:
# Run Scanorama
scanorama.integrate_scanpy(adata_sub, dimred=200, knn=50, verbose=True)

# Concatenate scanorama output 
X_scanorama = [ad.obsm['X_scanorama'] for ad in adata_sub]
X_scanorama = np.concatenate(X_scanorama)

obs_names = [ad.obs_names for ad in adata_sub]
obs_names = np.concatenate(obs_names)
all(obs_names==adata.obs_names)

# Add X_scanorama integration to adata 
adata.obsm["X_scanorama"] = X_scanorama

Found 2000 genes among all datasets
Processing datasets (0, 1)
In [27]:
# # Dimensional reduction and clustering 
sc.pp.neighbors(adata, n_neighbors=100, use_rep='X_scanorama')
sc.tl.leiden(adata, resolution=1)
sc.tl.louvain(adata, resolution=1)
sc.tl.umap(adata)

# Plot 
sc.pl.umap(adata, color=['louvain', 'leiden', 'tissue', 'treatment', 'label_fine_haemosphere', 'sample_rep', 'cc_phase_class', 'pHb_RNA', 'pRb_RNA'], wspace=0.5, ncols=3)
In [28]:
adata.write_h5ad('data/object/adata_sct_hvg2000.h5ad')
In [29]:
import sys
sys.path.insert(0, '../scFacility/script')
from dirFacility import adata2dir
In [30]:
adata2dir(adata, 'data/object/adata_sct_hvg2000/', assay="RNA", layers="counts", build_dir=True, overwrite=True)

Creating output directorydata/object/adata_sct_hvg2000/
Writing meta data: data/object/adata_sct_hvg2000/meta/meta.csv
Writing assay data for the layers:
Writing data for the reduction:
...  X_scanorama
...  X_umap

Scanorama (SCTransform + regression)¶

In [31]:
adata = sc.read_h5ad('data/object/so_sct_reg.h5ad')
In [32]:
def set_color(categories): 
    
    categories = [x for x in categories if x in list(adata.obs.columns)]

    for category in categories: 
        
        adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
        
        keys = list(color[category].keys())
        keys = [x for x in keys if x in list(adata.obs[category])]

        adata.obs[category] = adata.obs[category].cat.reorder_categories(keys)
        adata.uns[category+'_colors'] = np.array([color[category].get(key) for key in keys], dtype=object)
        
# Set colors
set_color(list(color.keys()))
In [33]:
adata_sub = dict()
for sample_group in adata.obs['treatment'].unique():
    adata_sub[sample_group] = adata[adata.obs['treatment']==sample_group].copy()
adata_sub = list(adata_sub.values())
In [34]:
# Run Scanorama
scanorama.integrate_scanpy(adata_sub, dimred=200, knn=50, verbose=True)

# Concatenate scanorama output 
X_scanorama = [ad.obsm['X_scanorama'] for ad in adata_sub]
X_scanorama = np.concatenate(X_scanorama)

obs_names = [ad.obs_names for ad in adata_sub]
obs_names = np.concatenate(obs_names)
all(obs_names==adata.obs_names)

# Add X_scanorama integration to adata 
adata.obsm["X_scanorama"] = X_scanorama

Found 8000 genes among all datasets
Processing datasets (0, 1)
In [35]:
# # Dimensional reduction and clustering 
sc.pp.neighbors(adata, n_neighbors=100, use_rep='X_scanorama')
sc.tl.leiden(adata, resolution=1)
sc.tl.louvain(adata, resolution=1)
sc.tl.umap(adata)

# Plot 
sc.pl.umap(adata, color=['louvain', 'leiden', 'tissue', 'treatment', 'label_fine_haemosphere', 'sample_rep', 'cc_phase_class', 'pHb_RNA', 'pRb_RNA'], wspace=0.5, ncols=3)

Scanorama (SCTransform + regression + 8000 HVG)¶

In [36]:
adata = sc.read_h5ad('data/object/so_sct_reg.h5ad')
In [37]:
hvg_8000 = list(adata.uns['hvg_int_8000'])
adata = adata[:,hvg_8000]
In [38]:
def set_color(categories): 
    
    categories = [x for x in categories if x in list(adata.obs.columns)]

    for category in categories: 
        
        adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
        
        keys = list(color[category].keys())
        keys = [x for x in keys if x in list(adata.obs[category])]

        adata.obs[category] = adata.obs[category].cat.reorder_categories(keys)
        adata.uns[category+'_colors'] = np.array([color[category].get(key) for key in keys], dtype=object)
        
# Set colors
set_color(list(color.keys()))

/tmp/ipykernel_3028845/1707382715.py:7: ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.
  adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
In [39]:
adata_sub = dict()
for sample_group in adata.obs['treatment'].unique():
    adata_sub[sample_group] = adata[adata.obs['treatment']==sample_group].copy()
adata_sub = list(adata_sub.values())
In [40]:
# Run Scanorama
scanorama.integrate_scanpy(adata_sub, dimred=200, knn=50, verbose=True)

# Concatenate scanorama output 
X_scanorama = [ad.obsm['X_scanorama'] for ad in adata_sub]
X_scanorama = np.concatenate(X_scanorama)

obs_names = [ad.obs_names for ad in adata_sub]
obs_names = np.concatenate(obs_names)
all(obs_names==adata.obs_names)

# Add X_scanorama integration to adata 
adata.obsm["X_scanorama"] = X_scanorama

Found 8000 genes among all datasets
Processing datasets (0, 1)
In [41]:
# Dimensional reduction and clustering 
sc.pp.neighbors(adata, n_neighbors=100, use_rep='X_scanorama')
sc.tl.leiden(adata, resolution=1)
sc.tl.louvain(adata, resolution=1)
sc.tl.umap(adata)

# Plot 
sc.pl.umap(adata, color=['louvain', 'leiden', 'tissue', 'treatment', 'label_fine_haemosphere', 'sample_rep', 'cc_phase_class', 'pHb_RNA', 'pRb_RNA'], wspace=0.5, ncols=3)

Scanorama (SCTransform + regression + 4000 HVG)¶

In [42]:
adata = sc.read_h5ad('data/object/so_sct_reg.h5ad')
In [43]:
hvg_4000 = list(adata.uns['hvg_int_4000'])
adata = adata[:,hvg_4000]
In [44]:
def set_color(categories): 
    
    categories = [x for x in categories if x in list(adata.obs.columns)]

    for category in categories: 
        
        adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
        
        keys = list(color[category].keys())
        keys = [x for x in keys if x in list(adata.obs[category])]

        adata.obs[category] = adata.obs[category].cat.reorder_categories(keys)
        adata.uns[category+'_colors'] = np.array([color[category].get(key) for key in keys], dtype=object)
        
# Set colors
set_color(list(color.keys()))

/tmp/ipykernel_3028845/1707382715.py:7: ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.
  adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
In [45]:
adata_sub = dict()
for sample_group in adata.obs['treatment'].unique():
    adata_sub[sample_group] = adata[adata.obs['treatment']==sample_group].copy()
adata_sub = list(adata_sub.values())
In [46]:
# Run Scanorama
scanorama.integrate_scanpy(adata_sub, dimred=200, knn=50, verbose=True)

# Concatenate scanorama output 
X_scanorama = [ad.obsm['X_scanorama'] for ad in adata_sub]
X_scanorama = np.concatenate(X_scanorama)

obs_names = [ad.obs_names for ad in adata_sub]
obs_names = np.concatenate(obs_names)
all(obs_names==adata.obs_names)

# Add X_scanorama integration to adata 
adata.obsm["X_scanorama"] = X_scanorama

Found 4000 genes among all datasets
Processing datasets (0, 1)
In [47]:
# # Dimensional reduction and clustering 
sc.pp.neighbors(adata, n_neighbors=100, use_rep='X_scanorama')
sc.tl.leiden(adata, resolution=1)
sc.tl.louvain(adata, resolution=1)
sc.tl.umap(adata)

# Plot 
sc.pl.umap(adata, color=['louvain', 'leiden', 'tissue', 'treatment', 'label_fine_haemosphere', 'sample_rep', 'cc_phase_class', 'pHb_RNA', 'pRb_RNA'], wspace=0.5, ncols=3)

Scanorama (SCTransform + regression + 2000 HVG)¶

In [48]:
adata = sc.read_h5ad('data/object/so_sct_reg.h5ad')
In [49]:
hvg_2000 = list(adata.uns['hvg_int_2000'])
adata = adata[:,hvg_2000]
In [50]:
def set_color(categories): 
    
    categories = [x for x in categories if x in list(adata.obs.columns)]

    for category in categories: 
        
        adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
        
        keys = list(color[category].keys())
        keys = [x for x in keys if x in list(adata.obs[category])]

        adata.obs[category] = adata.obs[category].cat.reorder_categories(keys)
        adata.uns[category+'_colors'] = np.array([color[category].get(key) for key in keys], dtype=object)
        
# Set colors
set_color(list(color.keys()))

/tmp/ipykernel_3028845/1707382715.py:7: ImplicitModificationWarning: Trying to modify attribute `.obs` of view, initializing view as actual.
  adata.obs[category] = pd.Series(adata.obs[category], dtype='category')
In [51]:
adata_sub = dict()
for sample_group in adata.obs['treatment'].unique():
    adata_sub[sample_group] = adata[adata.obs['treatment']==sample_group].copy()
adata_sub = list(adata_sub.values())
In [52]:
# Run Scanorama
scanorama.integrate_scanpy(adata_sub, dimred=200, knn=50, verbose=True)

# Concatenate scanorama output 
X_scanorama = [ad.obsm['X_scanorama'] for ad in adata_sub]
X_scanorama = np.concatenate(X_scanorama)

obs_names = [ad.obs_names for ad in adata_sub]
obs_names = np.concatenate(obs_names)
all(obs_names==adata.obs_names)

# Add X_scanorama integration to adata 
adata.obsm["X_scanorama"] = X_scanorama

Found 2000 genes among all datasets
Processing datasets (0, 1)
In [53]:
# # Dimensional reduction and clustering 
sc.pp.neighbors(adata, n_neighbors=100, use_rep='X_scanorama')
sc.tl.leiden(adata, resolution=1)
sc.tl.louvain(adata, resolution=1)
sc.tl.umap(adata)

# Plot 
sc.pl.umap(adata, color=['louvain', 'leiden', 'tissue', 'treatment', 'label_fine_haemosphere', 'sample_rep', 'cc_phase_class', 'pHb_RNA', 'pRb_RNA'], wspace=0.5, ncols=3)

Save output¶

In [54]:
adata.write_h5ad('data/object/adata_sct_reg_hvg2000.h5ad')
In [55]:
import sys
sys.path.insert(0, '../scFacility/script')
from dirFacility import adata2dir
In [56]:
adata2dir(adata, 'data/object/adata_sct_reg_hvg2000/', assay="RNA", layers="counts", build_dir=True, overwrite=True)

Creating output directorydata/object/adata_sct_reg_hvg2000/
Writing meta data: data/object/adata_sct_reg_hvg2000/meta/meta.csv
Writing assay data for the layers:
Writing data for the reduction:
...  X_scanorama
...  X_umap